import random
import time
import requests
import os
from bs4 import BeautifulSoup


# 抓取四大名著


def get_html(main_url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
    }
    """    获取指定URL的网页源代码
    """
    resp = requests.get(main_url, headers=headers)
    resp.encoding = resp.apparent_encoding
    soup = BeautifulSoup(resp.text, 'lxml')
    # print(soup)
    return soup


def get_book(soup):
    """      获取四大名著的书名和相应的url
    param soup
    return
    """
    # class_加下划线作为一个关键字参数来指定类名，不加会报错
    div = soup.find_all('div', class_="book-item")
    book_dic = {}  # 存储在字典中 {书名 ：书链接url}
    for con in div:
        # 获取书的链接
        book_name = con.get_text().replace('\n', '')
        # 获取书的链接url
        book_href = 'https://www.shicimingju.com' + con.a['href']
        book_dic[book_name] = book_href
    # print(book_dic)
    return book_dic


def get_book_mulu(books_html):
    """
    获取每本书的章节标题和URL
    param 当前章节的html内容
    return
    """
    # 包含了整个章节的div
    div = books_html.find_all('div', class_="book-mulu")
    mulu_dict = {}  # 存储在字典中 {章节标题 ：章节链接url}
    for mulu in div:
        # 抓取超链接
        mulu_hrefs = mulu.find_all('a')
        for mulu_href in mulu_hrefs:
            title = mulu_href.get_text()
            href = mulu_href['href']
            mulu_dict[title] = 'https://www.shicimingju.com' + href
        # print(mulu_dict)
        return mulu_dict


def book_mulu_content(chapter, books_html):
    """获取章节内容
    param1 chapter 章节标题
    param2 books_html 获取到的bs4对象的页面内容
    return
    """
    con_dic = {}  # 存储在字典中 {章节标题 ：章节内容}
    div = books_html.find('div', class_='chapter_content')
    text = div.text
    con_dic[chapter] = text
    # print(con_dic)
    return con_dic


def save_books(book_name, book_contents):
    """
    将获取到的章节内容保存到本地文件
    param1 book_name 四大名著的书名
    param2 book_contents 抓取到的章节里面的内容
    return
    """
    # 创建目录
    if not os.path.exists(book_name):
        os.mkdir(book_name)
    for title in book_contents:
        path = os.path.join(book_name, title + '.text')
        with open(path, 'a', encoding='utf-8') as f:
            f.write(book_contents[title])
            print(f'{book_name} === {title}下载完成!!!')


def main(main_url):
    """
    运行的主函数
    param main_url 链接
    return
    """
    book_dic = get_book(get_html(main_url))
    # 循环获取到书名，得到书名后就能得到url
    for book_name in book_dic:
        # 得到四大名著的章节信息
        mulu_dic = get_book_mulu(get_html(book_dic[book_name]))
        for title, url in mulu_dic.items():
            # 输出章节标题和对应url
            # print(title,url)
            # 获取章节内容
            book_contents = book_mulu_content(title, get_html(url))
            save_books(book_name, book_contents)
            time.sleep(random.randint(1, 3))
            # 运行一次就退出
            # exit()


if __name__ == '__main__':
    main_url = 'https://www.shicimingju.com/bookmark/sidamingzhu.html'
    main(main_url)
